#!/usr/local/bin/python
# -*-encoding:utf-8 -*-
import numpy as np;
import scipy.sparse as sp;
import time
import threading
import os
from multiprocessing import Queue,Process,Manager
import profile
import utils

#For this time, Return csc_matrix
def getColumnVector(s, d):
    row = [];
    col = [];
    data = [];
    li = s.strip().split(' ');
    for l in li:
        pair = l.strip().split(':');
        index = int(pair[0]);
        value = float(pair[1]);
        row.append(index-1);
        col.append(0);
        data.append(value)
    return sp.csc_matrix((data,(row, col)), shape=(d,1));


def saveTemp():
    utils.save_sparse_csc('Wps.txt', W);
    utils.save_sparse_csc('Nps.txt', N);

def computeWN(d, pid, start, end, debug=False):
    print pid, start, end;
    W = sp.csc_matrix((dy,dx), dtype=np.float64);
    N = sp.csc_matrix((dx, dy), dtype=np.float64);
    for i in range(start, end):
        if debug:
            print 'Debug: ', i;
        res, xi, yij,niy = di[i];
        W  = W + res/niy * yij.dot(xi.transpose());
        N = N + res/niy *  xi.dot(yij.transpose());

    d[pid] = (W, N);
    print pid, 'End';
#compute M and N batch

di = [];
def batchPreprocessing(featureDataFile):
    K = 20;
    featureData = open(featureDataFile, 'r');
    ni = 0;
    line = featureData.readline();
    if not line.startswith('*'):
        print 'No Dimension Provided';
        return;
    global dx, dy, W ,N;
    firstLi = line.split('\t');
    dx = int(firstLi[1]) ;
    dy = int(firstLi[2]);
    print 'dimension:', dx, dy;

    linec = 1;
    global W,N;
    W = sp.csc_matrix((dy, dx), dtype=np.float64);
    N = sp.csc_matrix((dx, dy), dtype=np.float64);
    while(True):
        linec += 1;
        print 'processing %d' % linec;
        line = featureData.readline();
        if not line:
            break;
        elif line.startswith('#'):
            li = line.split('\t');
            i = li[1];
            ni += 1;
            niy = int(li[2])
            
            for j in range(niy):
                    pair = featureData.readline();
                    instance = pair.split('\t');
                    res = float(instance[0]);
                    if instance[1].strip() == '' or instance[2].strip() =='' or instance[2].strip() == 'None':
                        continue; 
                    xi = getColumnVector(instance[1].strip(), dx);
                    yij = getColumnVector(instance[2].strip(), dy);

                    di.append((res, xi, yij, niy));



    l = len(di);
    print l;
    ran = l / K;

    processList =[];
    index = 0;
    manager = Manager();
    
    d = manager.dict();
    for pid in range(K-1):
        processList.append(Process(target=computeWN, args=(d, pid, index , index+ran)));
        index = index + ran;

    processList.append(Process(target=computeWN, args=(d, K-1, index, l, True)));
    
    for p in processList:
        p.start();
    for p in processList:
        p.join();
    print d.keys();

    for pair in d.values():
        W  = W + pair[0];
        N = N + pair[1];
    W = W /float(ni);
    N = N /float(ni);
    print W;
    saveTemp();
    featureData.close();


if __name__ == '__main__':
    featureDataFile = '../train.set';
    batchPreprocessing(featureDataFile);
    
